# coding = utf-8
import copy
import string
import random
import xlwings as xw
from datetime import *
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from time import sleep
#import pytesseract
#from PIL import Image
import time
import datetime
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.action_chains import ActionChains
from fake_useragent import UserAgent
import selenium.common.exceptions

def BrowserInit():
    chrome_option = webdriver.ChromeOptions()
    chrome_option.add_experimental_option('useAutomationExtension',False)
    chrome_option.add_argument("disable-blink-features")
    # chrome_option.add_argument("headless")
    chrome_option.add_argument("disable-blink-features=AutomationControlled")
    chrome_option.add_argument = ("user_agent = Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.119 Safari/537.36") # 98.0.4758.80  106.0.5249.119
    chrome_option.add_experimental_option('excludeSwitches', ['enable-automation'])
    # prefs = {'profile.managed_default_content_settings.images': 2}
    # chrome_option.add_experimental_option('prefs',prefs)
    return chrome_option

def openFileAsin():
    asin_list = []
    with open(r'C:\Users\Administrator\Desktop\asin_list.txt', 'r') as f: # 打开要抓取的asin列表文档，并存储进asin_list
        for line in f:
            asin_list.append(line)
        yerdpt = [ok_asin.replace('\n','') for ok_asin in asin_list] # 删除每一行后面的换行字符
        print(yerdpt) # 打印要记录的asin列表
    return yerdpt
    
def change_UA(ChromeVerList,Counter):#传入Chrome版本号列表,和每次20的计数器，用于改变UA
    if Counter % 20 == 0:
        VersionNo = VersionNO_List[random.randint(0,len(VersionNO_List) - 1)]
        useragent = "user_agent = Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/" + VersionNo + " Safari/537.36"
        return  useragent
    else:
        pass


def dataHandle(data):
    for ADATE in data:
        if ADATE.isdigit():
            if len(ADATE) > 4:
                por = ADATE.split(",")
                s_connect = por[0]
                for ykfy in range(1,len(por)):
                    s_connect = s_connect + por[ykfy]
                sum_record = copy.deepcopy(int(s_connect))
            else:
                sum_record = copy.deepcopy(int(ADATE))
            break 
        else:
            pass   
    return sum_record

def saveWb(counter):
    if counter%50 == 0:
        wb.save(r'C:\Users\Administrator\Desktop\review.xlsx')

def element_Xpath(driver):
    circleno = ["womens top","sexy dress for women","summer sleeveless top for women","jumpsuit","dress","pencil box","computerpad",
                "glasscup","keyboard for desktop","laptop computer","sexy bra for summer","sexy bikini set","student toys","black t-back",
                "sexual toys","audlt tools","books for work","earphone HUAWEI","Huawei cellphone","Huawei routers","Huawei desktop computer",
                "2 pcs outfits fot ladies","sexy ladies clothes","mouse lapod","red wine china","MaoTai from china","sexy underware for ladie",
                "pornhub prime","file storage device","transparent underware","sexualnice pin","kawayi vergination"]
    sendkeytext = random.choice(circleno)
        
    #-------------------------------------鼠标事件---------------------------------------
    
    elementEvent = driver.find_element(By.XPATH,'//*[@id="twotabsearchtextbox"]')#top_search_box
    ActionChains(driver).click(elementEvent).send_keys(sendkeytext).perform()
    #elif num == 1:
    #    elementEvent = driver.find_element(By.XPATH,'//*[@id="a-page"]/div[3]/div/div[1]/div/div[2]/div[2]/form/div[1]/textarea')#ask_anwser_box
    #    ActionChains(driver).click(elementEvent).send_keys(sendkeytext).perform()
    #elif num == 2:
    #    elementEvent = driver.find_element(By.XPATH,'//*[@id="filterByKeywordTextBox"]')#review_search_box
    #    ActionChains(driver).click(elementEvent).send_keys(sendkeytext).perform()
    # return elementEvent

    #-------------------------------------鼠标事件---------------------------------------    

if __name__ == '__main__':
    chrome_option = BrowserInit()
    caps = DesiredCapabilities().CHROME
    caps["pageLoadStrategy"] = "none"
    
    # star_filter_condition = "/ref=cm_cr_arp_d_viewopt_sr?ie=UTF8&formatType=current_format&filterByStar="
    # 这个三个地址参数：head_url、star_filter_condition、page_num，地址可以是任意店铺任意一个asin的评论页面的地址，如果改地址无效，只要到亚马逊网站，任意查找一个asin的评论页面用以代替即可。
    # 只要保证最终的地址：url = head_url+asin+star_filter_condition+star_num+page_num中的asin是唯一的就行
    head_url = "https://www.amazon.com/"   #要抓取review的站点
    star_filter_condition = "/ref=cm_cr_arp_d_viewopt_sr?ie=UTF8&formatType=current_format&filterByStar="  # 抓取的星级过滤，要在等号后加几星
    page_num = "&pageNumber=1"  #页码
    title = ["ASIN", "5星", "4星", "3星", "2星", "1星"]  # 写入excel的列标题
    
    fnone = 0
    change_Version_Number_Count = 0
    # 为避免过多的被识别为机器人，随机选择一些卖家店铺的链接来抓取数据
    VersionNO_List = ["100.0.4896.127","93.0.4577.63","103.0.5060.66","97.0.4692.99","103.0.2060.53","101.0.4951.54",
					"100.0.4896.60","98.0.4758.102","99.0.4844.51","102.0.5005.115","100.0.4896.60","98.0.4758.80",
					"102.0.5005.115","100.0.4896.88"]
    Product_Title = ["Hanes-Womens-EcoSmart-Full-Zip-Sweatshirt","ZOWYA-Cap-Casual-Cap-Unisex-Adjustable-Size-Precurved-Profile-Unstructured-Fitted-Soft-Over",
                    "Sceptre-E248W-19203R-Monitor-Speakers-Metallic","Michael-Kors-Multifunction-Wristlet-Admiral",
                    "Electronics-Dual-Port-Charging-Included-EP-TA220NBEGUS","Etekcity-Bathroom-Bluetooth-Weighing-Compositions",
                    "TomCare-Storage-Organizer-Shelves-Shelving","Dr-Martens-Leona-Hook-Boot",
                    "Polaroid-Now-I-Type-Instant-Camera","ZESICA-Womens-Bohemian-Printed-Strapless",
                    "RAY-BAN-RB3548N-Hexagonal-Sunglasses-Gradient","SWAROVSKI-Womens-Infinity-Necklace-finish",
                    "Lanzom-Women-Straw-Panama-Fedora","Amazfit-Smartwatch-Customizable-Resistance-Activity",
                    "Apple-iPhone-Purple-Carrier-Subscription","Electric-Temperature-Control-Oilless-Dishwasher",
                    "Crocs-Classic-Comfortable-Casual-Medium","AmazonBasics-AA-Performance-Alkaline-Batteries",
                    "Xbox-Gift-Card-Digital-Code","Amazon-Brand-Gentle-Touch-Diapers",
                    "Gildan-Fleece-Hooded-Sweatshirt-G18500","PETLIBRO-Automatic-Dispenser-Indication-Clog-Free",
                    "Loomla-Bellflower-Drinking-Fountain-Dark","Oculus-Quest-Advanced-All-One-Virtual",
                    "TOZO-T12-Headphones-Intelligence-Waterproof","Backpack-Business-Charging-Resistant-Computer",
                    "Show-Socks-Women-Slip-Pairs","HP-Cartridge-F6U64AN-Deskjet-Officejet",
                    "Ergonomic-Compatible-Notebook-Soundance-LS1","AmazonBasics-Puresoft-PU-Padded-Mid-Back-Computer",
                    "SimpleHouseware-Organizer-Sliding-Upright-Sections","Amoretu-Womens-Sleeve-Tunics-Casual",
                    "Ru-Sweet-Waisted-Athletic-Sweatpants","MEROKEETY-Womens-Cardigan-Sweater-Outerwear",
                    "ANRABESS-Crewneck-Batwing-Oversized-Pullover","Kapoohott-Hoodies-Printed-Pullover-Sweatshirts",
                    "Halife-Leopard-Pullover-Sweatshirt-Pockets","KILIG-Womens-Sleeve-V-Neck-Button",
                    "Sloan Brands Personalized Miniature Steak Branding Iron","Penny Drop Game Small Version - A Fun Family or Bar Game",
                    "WOOSEA-Women-Sleeveless-Evening-Cocktail","Popilush-Shaper-Bodycon-Shapewear-Dresses","YMDUCH-Elegant-Sleeveless-Shoulder-Bodycon",
                    "Cosonsen-Womens-V-Neck-Sleeve-Dresses","GOKATOSAU-Womens-Elegant-Ruffle-Shoulder","PRETTYGARDEN-Womens-Casual-Bodycon-Pockets",
                    "GOBLES-Womens-Sleeve-Bodycon-Cocktail"]
    rows_cnt = 0
    real_cnt = []
    title_list = []
    star_list = ["five_star","four_star","three_star","two_star","one_star"] # 用于抓取星级的过滤
    one_asin_cnt = [] # 存储抓取到的asin的星级数据
    star_record = [] # 存储抓取到的数据的列表，用于之后的数据处理，筛选出想要的数据
    ERR_LIST = [] # 变狗的asin链接，记录未记录成功的asin
    real_total_record = [] # 存储某个asin总的review数量，如果为零，则直接跳过后面星级review的抓取，one_asin_cnt列表的数据直接用0填充
    title_list.append(title)
    asin_save_counter = 0
    end_data = int(time.mktime(time.strptime("2023-12-31", '%Y-%m-%d')))
    now_data = datetime.datetime.now().strftime('%Y-%m-%d')
    now_data = int(time.mktime(time.strptime(now_data, '%Y-%m-%d')))

    ASINS = openFileAsin()
    
    wb = xw.Book() # 打开excel
    sht = xw.sheets.active # 激活当前工作表
    sht.range('a1').value = title_list # 将列标题写入excel
    driver = webdriver.Chrome(options = chrome_option)
    starttime = time.time()
    if now_data < end_data:
        if "uk" in head_url: # 如果是英国站，暂停5秒，等待手动操作浏览器的cookie申请，否则不用等待
            asin_total_record_url = head_url + "GOBLES-Womens-Sleeve-Bodycon-Cocktail/product-reviews/B09JFM87BP/ref=cm_cr_arp_d_rvw_fmt?ie=UTF8&formatType=current_format"	 # 打开review链接																				
            driver.get(asin_total_record_url)
            sleep(5)
        else:
            pass
        for real_asin in ASINS: # 循环打开asin链接
            done = 0 
            one_asin_cnt.clear()
            asin_save_counter = asin_save_counter +1
            change_Version_Number_Count = change_Version_Number_Count + 1
            UA_RAND = change_UA(VersionNO_List,change_Version_Number_Count)
            chrome_option.add_argument = (UA_RAND)
            # driver = webdriver.Chrome(options = chrome_option)
            


            product_name_index = random.randint(1,len(Product_Title)) # 随机从Product_Title中选择一个店铺的链接作为浏览器打开的链接
            product_name = Product_Title [product_name_index - 1]            

            one_asin_cnt.append(real_asin)
            asin_total_record_url = head_url + product_name + "/product-reviews/" + real_asin + "/ref=cm_cr_arp_d_rvw_fmt?ie=UTF8&formatType=current_format"# 打开review链接																				
            driver.get(asin_total_record_url)
            try:
                driver.find_element(by = By.XPATH,value = '//*[@id="captchacharacters"]')
                sleep(10)
            except:
                pass
            try: # 获取asin的总review数
                try:
                    asin_total_record = driver.find_element(by = By.XPATH,value = '//*[@id="filter-info-section"]/div[2]')   
                except:
                    asin_total_record = driver.find_element(by = By.XPATH,value = '//*[@id="filter-info-section"]/div')
                total_record = asin_total_record.get_attribute('textContent')
            except: # 如果没有找到则认为是变狗的asin            
                ERR_LIST.append(copy.deepcopy(real_asin))
                one_asin_cnt.clear()
                continue	
            
            asd = total_record.split(" ") # 处理数据所在的行一遍获取需要的数据
            ASIN_START_VALUE = dataHandle(asd)
            if ASIN_START_VALUE == 0: # asin的review为0时，跳过之后的数据获取，直接填0
                for oskg in range(0, 5):
                    one_asin_cnt.append(0)
                    print(one_asin_cnt)
            else: # asin的review不为0时，从5星开始获取review，直到1星
                for star_cnt in star_list:
                    star_num = str(star_cnt)
                    url = head_url + product_name + "/product-reviews/" + real_asin + star_filter_condition + star_num + page_num
                    driver.get(url)
                    try:
                        text = driver.find_element(by = By.XPATH,value = '//*[@id="filter-info-section"]/div[2]')
                        star_record = text.get_attribute('textContent')
                        a = star_record.split(" ") 
                        one_asin_cnt.append(dataHandle(a))
                        ASIN_START_VALUE = ASIN_START_VALUE-dataHandle(a)
                    except:
                        ERR_LIST.append(copy.deepcopy(real_asin))
                        one_asin_cnt.clear()
                        break
                    if len(one_asin_cnt) < 6: 
                        if ASIN_START_VALUE == 0:
                            print(one_asin_cnt)
                            for ouprt in range(len(one_asin_cnt),6):
                                one_asin_cnt.append(0)
                                print(one_asin_cnt)
                            break
                        else:
                            print(one_asin_cnt)
                    else:
                        print(one_asin_cnt)				
            if len(one_asin_cnt) > 0:
                real_cnt.append(copy.deepcopy(one_asin_cnt))
            else:
                pass
            
            # actionchainNo = random.randint(0,3)
            try:
                cdr = element_Xpath(driver)
            except:
                pass
            if len(real_cnt) > 0: # 将获取到的asin的数据写入excel
                record_cnt = 2 + rows_cnt
                sht.range('a' + str(record_cnt)).value = real_cnt
                rows_cnt = rows_cnt + 1
                real_cnt.clear()
            saveWb(asin_save_counter)#每记录50个asin保存一次
    else:
        pass
    print(real_cnt)
    print('以下Asin未记录成功:',ERR_LIST)
    sht.range('K1').value = ERR_LIST
    wb.save(r'C:\Users\Administrator\Desktop\review.xlsx')
    endtime = time.time()
    driver.quit()
    totaltime = (endtime - starttime) / 3600
    print("登记 {} 个asin,总耗时: {} hrs".format(len(ASINS),str(totaltime)))
    input('Press any key to quit program.')